{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "import pandas as pd\n",
    "import numpy as np\n",
    "import  matplotlib.pyplot as  plt\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>comment</th>\n",
       "      <th>star</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>还行吧，建议不要排队那个烤鸭和羊肉串，因为烤肉时间本来就不够，排那个要半小时，然后再回来吃烤...</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>去过好几次了 东西还是老样子 没增添什么新花样 环境倒是挺不错 离我们这也挺近 味道还可以 ...</td>\n",
       "      <td>40</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>一个字：好！！！ #羊肉串# #五花肉# #牛舌# #很好吃# #鸡软骨# #拌菜# #抄河...</td>\n",
       "      <td>50</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>第一次来吃，之前看过好多推荐说这个好吃，真的抱了好大希望，排队的人挺多的，想吃得趁早来啊。还...</td>\n",
       "      <td>20</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>羊肉串真的不太好吃，那种说膻不膻说臭不臭的味。烤鸭还行，大虾没少吃，也就到那吃大虾了，吃完了...</td>\n",
       "      <td>30</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                             comment  star\n",
       "0  还行吧，建议不要排队那个烤鸭和羊肉串，因为烤肉时间本来就不够，排那个要半小时，然后再回来吃烤...    40\n",
       "1  去过好几次了 东西还是老样子 没增添什么新花样 环境倒是挺不错 离我们这也挺近 味道还可以 ...    40\n",
       "2  一个字：好！！！ #羊肉串# #五花肉# #牛舌# #很好吃# #鸡软骨# #拌菜# #抄河...    50\n",
       "3  第一次来吃，之前看过好多推荐说这个好吃，真的抱了好大希望，排队的人挺多的，想吃得趁早来啊。还...    20\n",
       "4  羊肉串真的不太好吃，那种说膻不膻说臭不臭的味。烤鸭还行，大虾没少吃，也就到那吃大虾了，吃完了...    30"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df=pd.read_excel(\"all_data_meituan.xlsx\")[[\"comment\",\"star\"]]\n",
    "df.head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(17400, 2)"
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "df.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [],
   "source": [
    "df['sentiment']=df['star'].apply(lambda x:1 if x>30 else 0)\n",
    "df=df.drop_duplicates() ## 去掉重复的评论\n",
    "df=df.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3138, 1)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "X=pd.concat([df[['comment']],df[['comment']],df[['comment']]])\n",
    "y=pd.concat([df.sentiment,df.sentiment,df.sentiment])\n",
    "X.columns=['comment']\n",
    "X.reset_index\n",
    "X.shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Building prefix dict from the default dictionary ...\n",
      "Loading model from cache C:\\Users\\HUANG_~1\\AppData\\Local\\Temp\\jieba.cache\n",
      "Loading model cost 1.051 seconds.\n",
      "Prefix dict has been built succesfully.\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0    还行 吧 ， 建议 不要 排队 那个 烤鸭 和 羊肉串 ， 因为 烤肉 时间 本来 就 不够...\n",
       "1    去过 好 几次 了   东西 还是 老 样子   没 增添 什么 新花样   环境 倒 是 ...\n",
       "2    一个 字 ： 好 ！ ！ ！   # 羊肉串 #   # 五花肉 #   # 牛舌 #   ...\n",
       "3    第一次 来 吃 ， 之前 看过 好多 推荐 说 这个 好吃 ， 真的 抱 了 好 大 希望 ...\n",
       "4    羊肉串 真的 不太 好吃 ， 那种 说 膻 不 膻 说 臭 不 臭 的 味 。 烤鸭 还 行...\n",
       "Name: cut_comment, dtype: object"
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import jieba\n",
    "def chinese_word_cut(mytext):\n",
    "    return \" \".join(jieba.cut(mytext))\n",
    "X['cut_comment']=X[\"comment\"].apply(chinese_word_cut)\n",
    "X['cut_comment'].head()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import  train_test_split\n",
    "X_train,X_test,y_train,y_test= train_test_split(X,y,random_state=42,test_size=0.25)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_custom_stopwords(stop_words_file):\n",
    "    with open(stop_words_file,encoding=\"utf-8\") as f:\n",
    "        custom_stopwords_list=[i.strip() for i in f.readlines()]\n",
    "    return custom_stopwords_list"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['100', '01', '02', '03', '04', '05', '06', '07', '08', '09']"
      ]
     },
     "execution_count": 9,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "stop_words_file = \"stopwords.txt\"\n",
    "stopwords = get_custom_stopwords(stop_words_file)\n",
    "stopwords[-10:]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None, stop_words=None,\n",
       "        strip_accents=None, token_pattern='(?u)\\\\b\\\\w\\\\w+\\\\b',\n",
       "        tokenizer=None, vocabulary=None)"
      ]
     },
     "execution_count": 10,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.feature_extraction.text import  CountVectorizer\n",
    "vect=CountVectorizer()\n",
    "vect"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<2353x1965 sparse matrix of type '<class 'numpy.int64'>'\n",
       "\twith 20491 stored elements in Compressed Sparse Row format>"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vect.fit_transform(X_train[\"cut_comment\"])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(2353, 1965)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vect.fit_transform(X_train[\"cut_comment\"]).toarray().shape"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "# pd.DataFrame(vect.fit_transform(X_train[\"cut_comment\"]).toarray(),columns=vect.get_feature_names()).iloc[:10,:22]\n",
    "# print(vect.get_feature_names())\n",
    "# #  数据维数1956，不算很大（未使用停用词）"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>amazing</th>\n",
       "      <th>happy</th>\n",
       "      <th>ktv</th>\n",
       "      <th>pm2</th>\n",
       "      <th>一万个</th>\n",
       "      <th>一个多</th>\n",
       "      <th>一个月</th>\n",
       "      <th>一串</th>\n",
       "      <th>一人</th>\n",
       "      <th>一件</th>\n",
       "      <th>...</th>\n",
       "      <th>麻烦</th>\n",
       "      <th>麻酱</th>\n",
       "      <th>黄喉</th>\n",
       "      <th>黄桃</th>\n",
       "      <th>黄花鱼</th>\n",
       "      <th>黄金</th>\n",
       "      <th>黑乎乎</th>\n",
       "      <th>黑椒</th>\n",
       "      <th>黑胡椒</th>\n",
       "      <th>齐全</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>...</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "<p>5 rows × 1691 columns</p>\n",
       "</div>"
      ],
      "text/plain": [
       "   amazing  happy  ktv  pm2  一万个  一个多  一个月  一串  一人  一件 ...  麻烦  麻酱  黄喉  黄桃  \\\n",
       "0        0      0    0    0    0    0    0   0   0   0 ...   0   0   0   0   \n",
       "1        0      0    0    0    0    0    0   0   0   0 ...   0   0   0   0   \n",
       "2        0      0    0    0    0    0    0   0   0   0 ...   0   0   0   0   \n",
       "3        0      0    0    0    0    0    0   0   0   0 ...   0   0   0   0   \n",
       "4        0      0    0    0    0    0    0   0   0   0 ...   0   0   0   0   \n",
       "\n",
       "   黄花鱼  黄金  黑乎乎  黑椒  黑胡椒  齐全  \n",
       "0    0   0    0   0    0   0  \n",
       "1    0   0    0   0    0   0  \n",
       "2    0   0    0   0    0   0  \n",
       "3    0   0    0   0    0   0  \n",
       "4    0   0    0   0    0   0  \n",
       "\n",
       "[5 rows x 1691 columns]"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "vect = CountVectorizer(token_pattern=u'(?u)\\\\b[^\\\\d\\\\W]\\\\w+\\\\b',stop_words=frozenset(stopwords)) # 去除停用词\n",
    "pd.DataFrame(vect.fit_transform(X_train['cut_comment']).toarray(), columns=vect.get_feature_names()).head()\n",
    "# 1691 columns,去掉以数字为特征值的列，减少了三列编程1691 \n",
    "# max_df = 0.8 # 在超过这一比例的文档中出现的关键词（过于平凡），去除掉。\n",
    "# min_df = 3 # 在低于这一数量的文档中出现的关键词（过于独特），去除掉。"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('countvectorizer', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None,\n",
       "        stop_words=...,\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])"
      ]
     },
     "execution_count": 15,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.pipeline import make_pipeline\n",
    "from sklearn.svm import SVC\n",
    "from sklearn import  metrics\n",
    "svc_cl=SVC()\n",
    "pipe=make_pipeline(vect,svc_cl)\n",
    "pipe.fit(X_train.cut_comment, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.63184713375796175"
      ]
     },
     "execution_count": 16,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[  0, 289],\n",
       "       [  0, 496]], dtype=int64)"
      ]
     },
     "execution_count": 17,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  支持向量机分类"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.63184713375796175"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.svm import SVC\n",
    "svc_cl=SVC() # 实例化\n",
    "pipe=make_pipeline(vect,svc_cl)\n",
    "pipe.fit(X_train.cut_comment, y_train)\n",
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 支持向量机 网格搜索"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.model_selection import GridSearchCV\n",
    "from sklearn.svm import SVC\n",
    "from sklearn.pipeline import  Pipeline\n",
    "# svc=SVC(random_state=1)\n",
    "from sklearn.linear_model import SGDClassifier\n",
    "from sklearn.feature_extraction.text import TfidfTransformer\n",
    "tfidf=TfidfTransformer()\n",
    "# ('tfidf',\n",
    "#                       TfidfTransformer()),\n",
    "#                      ('clf',\n",
    "#                       SGDClassifier(max_iter=1000)),\n",
    "# svc=SGDClassifier(max_iter=1000)\n",
    "svc=SVC()\n",
    "# pipe=make_pipeline(vect,SVC)\n",
    "pipe_svc=Pipeline([(\"scl\",vect),('tfidf',tfidf),(\"clf\",svc)])\n",
    "para_range=[0.0001,0.001,0.01,0.1,1.0,10,100,1000]\n",
    "para_grid=[\n",
    "    {'clf__C':para_range,\n",
    "    'clf__kernel':['linear']},\n",
    "    {'clf__gamma':para_range,\n",
    "    'clf__kernel':['rbf']}\n",
    "]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "gs=GridSearchCV(estimator=pipe_svc,param_grid=para_grid,cv=10,n_jobs=-1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "GridSearchCV(cv=10, error_score='raise',\n",
       "       estimator=Pipeline(memory=None,\n",
       "     steps=[('scl', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None,\n",
       "        stop_words=frozenset({'...,\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))]),\n",
       "       fit_params=None, iid=True, n_jobs=-1,\n",
       "       param_grid=[{'clf__C': [0.0001, 0.001, 0.01, 0.1, 1.0, 10, 100, 1000], 'clf__kernel': ['linear']}, {'clf__gamma': [0.0001, 0.001, 0.01, 0.1, 1.0, 10, 100, 1000], 'clf__kernel': ['rbf']}],\n",
       "       pre_dispatch='2*n_jobs', refit=True, return_train_score='warn',\n",
       "       scoring=None, verbose=0)"
      ]
     },
     "execution_count": 21,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.fit(X_train.cut_comment,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('scl', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None,\n",
       "        stop_words=frozenset({'...,\n",
       "  max_iter=-1, probability=False, random_state=None, shrinking=True,\n",
       "  tol=0.001, verbose=False))])"
      ]
     },
     "execution_count": 22,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "gs.best_estimator_.fit(X_train.cut_comment,y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95031847133757963"
      ]
     },
     "execution_count": 23,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = gs.best_estimator_.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 临近法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Pipeline(memory=None,\n",
       "     steps=[('countvectorizer', CountVectorizer(analyzer='word', binary=False, decode_error='strict',\n",
       "        dtype=<class 'numpy.int64'>, encoding='utf-8', input='content',\n",
       "        lowercase=True, max_df=1.0, max_features=None, min_df=1,\n",
       "        ngram_range=(1, 1), preprocessor=None,\n",
       "        stop_words=...owski',\n",
       "           metric_params=None, n_jobs=1, n_neighbors=4, p=2,\n",
       "           weights='uniform'))])"
      ]
     },
     "execution_count": 24,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.neighbors import  KNeighborsClassifier\n",
    "knn=KNeighborsClassifier(n_neighbors=4,p=2,metric='minkowski')\n",
    "pipe=make_pipeline(vect,knn)\n",
    "pipe.fit(X_train.cut_comment, y_train)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.81910828025477711"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[188, 101],\n",
       "       [ 41, 455]], dtype=int64)"
      ]
     },
     "execution_count": 26,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 决策树"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {},
   "outputs": [],
   "source": [
    "from sklearn.tree import DecisionTreeClassifier\n",
    "tree=DecisionTreeClassifier(criterion='entropy',random_state=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.93885350318471339"
      ]
     },
     "execution_count": 28,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "pipe=make_pipeline(vect,tree)\n",
    "pipe.fit(X_train.cut_comment, y_train)\n",
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[256,  33],\n",
       "       [ 15, 481]], dtype=int64)"
      ]
     },
     "execution_count": 29,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### 随机森林"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.96560509554140128"
      ]
     },
     "execution_count": 30,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from sklearn.ensemble import RandomForestClassifier\n",
    "forest=RandomForestClassifier(criterion='entropy',random_state=1,n_jobs=2)\n",
    "pipe=make_pipeline(vect,forest)\n",
    "pipe.fit(X_train.cut_comment, y_train)\n",
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)\n",
    "# 加上tfidf反而准确率96.5至95.0，"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[265,  24],\n",
       "       [  3, 493]], dtype=int64)"
      ]
     },
     "execution_count": 31,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### bagging方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95541401273885351"
      ]
     },
     "execution_count": 32,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "\n",
    "from sklearn.ensemble import BaggingClassifier\n",
    "bag=BaggingClassifier(base_estimator=tree,\n",
    "                     n_estimators=10,\n",
    "                     max_samples=1.0,\n",
    "                     max_features=1.0,\n",
    "                     bootstrap=True,\n",
    "                     bootstrap_features=False,\n",
    "                     n_jobs=1,random_state=1)\n",
    "pipe=make_pipeline(vect,tfidf,bag)\n",
    "pipe.fit(X_train.cut_comment, y_train)\n",
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)  #  没用转化td-idf 93.2%, 加上转化步骤，准确率提升到95.5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[260,  29],\n",
       "       [  6, 490]], dtype=int64)"
      ]
     },
     "execution_count": 33,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "####  Gradient Boosting方法"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "GradientBoostingClassifier(criterion='friedman_mse', init=None,\n",
      "              learning_rate=0.18, loss='deviance', max_depth=10,\n",
      "              max_features='sqrt', max_leaf_nodes=None,\n",
      "              min_impurity_decrease=0.01, min_impurity_split=None,\n",
      "              min_samples_leaf=1, min_samples_split=2,\n",
      "              min_weight_fraction_leaf=0.0, n_estimators=240,\n",
      "              presort='auto', random_state=42, subsample=0.9, verbose=0,\n",
      "              warm_start=False)\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "0.96560509554140128"
      ]
     },
     "execution_count": 34,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from sklearn.ensemble import GradientBoostingClassifier\n",
    "grd = GradientBoostingClassifier(learning_rate=0.18,max_depth=10,n_estimators=240,random_state=42,max_features='sqrt',subsample=0.9,\n",
    "                                min_impurity_decrease=0.01)\n",
    "                                \n",
    "print(grd)\n",
    "# Choosing subsample < 1.0 leads to a reduction of variance and an increase in bias.\n",
    "# Choosing max_features < n_features leads to a reduction of variance and an increase in bias.降低过拟合，但是可能会增加偏差，降低方差（对应的过拟合）\n",
    "pipe=make_pipeline(vect,tfidf,grd)\n",
    "pipe.fit(X_train.cut_comment, y_train)\n",
    "y_pred = pipe.predict(X_test.cut_comment)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[265,  24],\n",
       "       [  3, 493]], dtype=int64)"
      ]
     },
     "execution_count": 35,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### xgboost 方法 sklearn类似 api"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 36,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[0]\tvalidation_0-error:0.295541\n",
      "Will train until validation_0-error hasn't improved in 200 rounds.\n",
      "[1]\tvalidation_0-error:0.277707\n",
      "[2]\tvalidation_0-error:0.267516\n",
      "[3]\tvalidation_0-error:0.24586\n",
      "[4]\tvalidation_0-error:0.23949\n",
      "[5]\tvalidation_0-error:0.236943\n",
      "[6]\tvalidation_0-error:0.243312\n",
      "[7]\tvalidation_0-error:0.235669\n",
      "[8]\tvalidation_0-error:0.226752\n",
      "[9]\tvalidation_0-error:0.233121\n",
      "[10]\tvalidation_0-error:0.234395\n",
      "[11]\tvalidation_0-error:0.233121\n",
      "[12]\tvalidation_0-error:0.230573\n",
      "[13]\tvalidation_0-error:0.225478\n",
      "[14]\tvalidation_0-error:0.219108\n",
      "[15]\tvalidation_0-error:0.210191\n",
      "[16]\tvalidation_0-error:0.203822\n",
      "[17]\tvalidation_0-error:0.198726\n",
      "[18]\tvalidation_0-error:0.197452\n",
      "[19]\tvalidation_0-error:0.192357\n",
      "[20]\tvalidation_0-error:0.189809\n",
      "[21]\tvalidation_0-error:0.193631\n",
      "[22]\tvalidation_0-error:0.188535\n",
      "[23]\tvalidation_0-error:0.189809\n",
      "[24]\tvalidation_0-error:0.183439\n",
      "[25]\tvalidation_0-error:0.184713\n",
      "[26]\tvalidation_0-error:0.178344\n",
      "[27]\tvalidation_0-error:0.174522\n",
      "[28]\tvalidation_0-error:0.169427\n",
      "[29]\tvalidation_0-error:0.170701\n",
      "[30]\tvalidation_0-error:0.164331\n",
      "[31]\tvalidation_0-error:0.165605\n",
      "[32]\tvalidation_0-error:0.165605\n",
      "[33]\tvalidation_0-error:0.164331\n",
      "[34]\tvalidation_0-error:0.156688\n",
      "[35]\tvalidation_0-error:0.161783\n",
      "[36]\tvalidation_0-error:0.161783\n",
      "[37]\tvalidation_0-error:0.161783\n",
      "[38]\tvalidation_0-error:0.156688\n",
      "[39]\tvalidation_0-error:0.155414\n",
      "[40]\tvalidation_0-error:0.16051\n",
      "[41]\tvalidation_0-error:0.159236\n",
      "[42]\tvalidation_0-error:0.16051\n",
      "[43]\tvalidation_0-error:0.16051\n",
      "[44]\tvalidation_0-error:0.161783\n",
      "[45]\tvalidation_0-error:0.159236\n",
      "[46]\tvalidation_0-error:0.15414\n",
      "[47]\tvalidation_0-error:0.151592\n",
      "[48]\tvalidation_0-error:0.152866\n",
      "[49]\tvalidation_0-error:0.149045\n",
      "[50]\tvalidation_0-error:0.146497\n",
      "[51]\tvalidation_0-error:0.150318\n",
      "[52]\tvalidation_0-error:0.150318\n",
      "[53]\tvalidation_0-error:0.143949\n",
      "[54]\tvalidation_0-error:0.138854\n",
      "[55]\tvalidation_0-error:0.13758\n",
      "[56]\tvalidation_0-error:0.135032\n",
      "[57]\tvalidation_0-error:0.133758\n",
      "[58]\tvalidation_0-error:0.13758\n",
      "[59]\tvalidation_0-error:0.129936\n",
      "[60]\tvalidation_0-error:0.127389\n",
      "[61]\tvalidation_0-error:0.129936\n",
      "[62]\tvalidation_0-error:0.133758\n",
      "[63]\tvalidation_0-error:0.133758\n",
      "[64]\tvalidation_0-error:0.133758\n",
      "[65]\tvalidation_0-error:0.129936\n",
      "[66]\tvalidation_0-error:0.127389\n",
      "[67]\tvalidation_0-error:0.129936\n",
      "[68]\tvalidation_0-error:0.129936\n",
      "[69]\tvalidation_0-error:0.126115\n",
      "[70]\tvalidation_0-error:0.123567\n",
      "[71]\tvalidation_0-error:0.127389\n",
      "[72]\tvalidation_0-error:0.121019\n",
      "[73]\tvalidation_0-error:0.118471\n",
      "[74]\tvalidation_0-error:0.115924\n",
      "[75]\tvalidation_0-error:0.115924\n",
      "[76]\tvalidation_0-error:0.115924\n",
      "[77]\tvalidation_0-error:0.115924\n",
      "[78]\tvalidation_0-error:0.115924\n",
      "[79]\tvalidation_0-error:0.115924\n",
      "[80]\tvalidation_0-error:0.112102\n",
      "[81]\tvalidation_0-error:0.112102\n",
      "[82]\tvalidation_0-error:0.115924\n",
      "[83]\tvalidation_0-error:0.112102\n",
      "[84]\tvalidation_0-error:0.117197\n",
      "[85]\tvalidation_0-error:0.11465\n",
      "[86]\tvalidation_0-error:0.11465\n",
      "[87]\tvalidation_0-error:0.100637\n",
      "[88]\tvalidation_0-error:0.103185\n",
      "[89]\tvalidation_0-error:0.109554\n",
      "[90]\tvalidation_0-error:0.109554\n",
      "[91]\tvalidation_0-error:0.110828\n",
      "[92]\tvalidation_0-error:0.110828\n",
      "[93]\tvalidation_0-error:0.110828\n",
      "[94]\tvalidation_0-error:0.109554\n",
      "[95]\tvalidation_0-error:0.104459\n",
      "[96]\tvalidation_0-error:0.104459\n",
      "[97]\tvalidation_0-error:0.104459\n",
      "[98]\tvalidation_0-error:0.103185\n",
      "[99]\tvalidation_0-error:0.109554\n",
      "[100]\tvalidation_0-error:0.107006\n",
      "[101]\tvalidation_0-error:0.105732\n",
      "[102]\tvalidation_0-error:0.101911\n",
      "[103]\tvalidation_0-error:0.103185\n",
      "[104]\tvalidation_0-error:0.095541\n",
      "[105]\tvalidation_0-error:0.095541\n",
      "[106]\tvalidation_0-error:0.098089\n",
      "[107]\tvalidation_0-error:0.096815\n",
      "[108]\tvalidation_0-error:0.095541\n",
      "[109]\tvalidation_0-error:0.094268\n",
      "[110]\tvalidation_0-error:0.095541\n",
      "[111]\tvalidation_0-error:0.095541\n",
      "[112]\tvalidation_0-error:0.092994\n",
      "[113]\tvalidation_0-error:0.098089\n",
      "[114]\tvalidation_0-error:0.098089\n",
      "[115]\tvalidation_0-error:0.094268\n",
      "[116]\tvalidation_0-error:0.092994\n",
      "[117]\tvalidation_0-error:0.089172\n",
      "[118]\tvalidation_0-error:0.087898\n",
      "[119]\tvalidation_0-error:0.082803\n",
      "[120]\tvalidation_0-error:0.08535\n",
      "[121]\tvalidation_0-error:0.08535\n",
      "[122]\tvalidation_0-error:0.08535\n",
      "[123]\tvalidation_0-error:0.080255\n",
      "[124]\tvalidation_0-error:0.081529\n",
      "[125]\tvalidation_0-error:0.084076\n",
      "[126]\tvalidation_0-error:0.084076\n",
      "[127]\tvalidation_0-error:0.082803\n",
      "[128]\tvalidation_0-error:0.084076\n",
      "[129]\tvalidation_0-error:0.084076\n",
      "[130]\tvalidation_0-error:0.084076\n",
      "[131]\tvalidation_0-error:0.082803\n",
      "[132]\tvalidation_0-error:0.082803\n",
      "[133]\tvalidation_0-error:0.082803\n",
      "[134]\tvalidation_0-error:0.086624\n",
      "[135]\tvalidation_0-error:0.086624\n",
      "[136]\tvalidation_0-error:0.082803\n",
      "[137]\tvalidation_0-error:0.082803\n",
      "[138]\tvalidation_0-error:0.082803\n",
      "[139]\tvalidation_0-error:0.082803\n",
      "[140]\tvalidation_0-error:0.082803\n",
      "[141]\tvalidation_0-error:0.082803\n",
      "[142]\tvalidation_0-error:0.082803\n",
      "[143]\tvalidation_0-error:0.082803\n",
      "[144]\tvalidation_0-error:0.082803\n",
      "[145]\tvalidation_0-error:0.082803\n",
      "[146]\tvalidation_0-error:0.082803\n",
      "[147]\tvalidation_0-error:0.082803\n",
      "[148]\tvalidation_0-error:0.082803\n",
      "[149]\tvalidation_0-error:0.082803\n",
      "[150]\tvalidation_0-error:0.082803\n",
      "[151]\tvalidation_0-error:0.082803\n",
      "[152]\tvalidation_0-error:0.082803\n",
      "[153]\tvalidation_0-error:0.082803\n",
      "[154]\tvalidation_0-error:0.082803\n",
      "[155]\tvalidation_0-error:0.082803\n",
      "[156]\tvalidation_0-error:0.080255\n",
      "[157]\tvalidation_0-error:0.080255\n",
      "[158]\tvalidation_0-error:0.082803\n",
      "[159]\tvalidation_0-error:0.082803\n",
      "[160]\tvalidation_0-error:0.082803\n",
      "[161]\tvalidation_0-error:0.08535\n",
      "[162]\tvalidation_0-error:0.082803\n",
      "[163]\tvalidation_0-error:0.084076\n",
      "[164]\tvalidation_0-error:0.087898\n",
      "[165]\tvalidation_0-error:0.087898\n",
      "[166]\tvalidation_0-error:0.087898\n",
      "[167]\tvalidation_0-error:0.08535\n",
      "[168]\tvalidation_0-error:0.084076\n",
      "[169]\tvalidation_0-error:0.081529\n",
      "[170]\tvalidation_0-error:0.081529\n",
      "[171]\tvalidation_0-error:0.081529\n",
      "[172]\tvalidation_0-error:0.081529\n",
      "[173]\tvalidation_0-error:0.081529\n",
      "[174]\tvalidation_0-error:0.080255\n",
      "[175]\tvalidation_0-error:0.082803\n",
      "[176]\tvalidation_0-error:0.082803\n",
      "[177]\tvalidation_0-error:0.082803\n",
      "[178]\tvalidation_0-error:0.084076\n",
      "[179]\tvalidation_0-error:0.081529\n",
      "[180]\tvalidation_0-error:0.081529\n",
      "[181]\tvalidation_0-error:0.077707\n",
      "[182]\tvalidation_0-error:0.080255\n",
      "[183]\tvalidation_0-error:0.080255\n",
      "[184]\tvalidation_0-error:0.077707\n",
      "[185]\tvalidation_0-error:0.077707\n",
      "[186]\tvalidation_0-error:0.077707\n",
      "[187]\tvalidation_0-error:0.077707\n",
      "[188]\tvalidation_0-error:0.075159\n",
      "[189]\tvalidation_0-error:0.075159\n",
      "[190]\tvalidation_0-error:0.075159\n",
      "[191]\tvalidation_0-error:0.075159\n",
      "[192]\tvalidation_0-error:0.075159\n",
      "[193]\tvalidation_0-error:0.075159\n",
      "[194]\tvalidation_0-error:0.075159\n",
      "[195]\tvalidation_0-error:0.075159\n",
      "[196]\tvalidation_0-error:0.075159\n",
      "[197]\tvalidation_0-error:0.075159\n",
      "[198]\tvalidation_0-error:0.075159\n",
      "[199]\tvalidation_0-error:0.075159\n",
      "[200]\tvalidation_0-error:0.075159\n",
      "[201]\tvalidation_0-error:0.071338\n",
      "[202]\tvalidation_0-error:0.071338\n",
      "[203]\tvalidation_0-error:0.06879\n",
      "[204]\tvalidation_0-error:0.071338\n",
      "[205]\tvalidation_0-error:0.071338\n",
      "[206]\tvalidation_0-error:0.06879\n",
      "[207]\tvalidation_0-error:0.071338\n",
      "[208]\tvalidation_0-error:0.06879\n",
      "[209]\tvalidation_0-error:0.06879\n",
      "[210]\tvalidation_0-error:0.072611\n",
      "[211]\tvalidation_0-error:0.072611\n",
      "[212]\tvalidation_0-error:0.072611\n",
      "[213]\tvalidation_0-error:0.072611\n",
      "[214]\tvalidation_0-error:0.072611\n",
      "[215]\tvalidation_0-error:0.072611\n",
      "[216]\tvalidation_0-error:0.072611\n",
      "[217]\tvalidation_0-error:0.072611\n",
      "[218]\tvalidation_0-error:0.072611\n",
      "[219]\tvalidation_0-error:0.072611\n",
      "[220]\tvalidation_0-error:0.072611\n",
      "[221]\tvalidation_0-error:0.072611\n",
      "[222]\tvalidation_0-error:0.072611\n",
      "[223]\tvalidation_0-error:0.072611\n",
      "[224]\tvalidation_0-error:0.072611\n",
      "[225]\tvalidation_0-error:0.075159\n",
      "[226]\tvalidation_0-error:0.072611\n",
      "[227]\tvalidation_0-error:0.072611\n",
      "[228]\tvalidation_0-error:0.072611\n",
      "[229]\tvalidation_0-error:0.075159\n",
      "[230]\tvalidation_0-error:0.075159\n",
      "[231]\tvalidation_0-error:0.072611\n",
      "[232]\tvalidation_0-error:0.06879\n",
      "[233]\tvalidation_0-error:0.072611\n",
      "[234]\tvalidation_0-error:0.072611\n",
      "[235]\tvalidation_0-error:0.071338\n",
      "[236]\tvalidation_0-error:0.072611\n",
      "[237]\tvalidation_0-error:0.06879\n",
      "[238]\tvalidation_0-error:0.071338\n",
      "[239]\tvalidation_0-error:0.071338\n",
      "[240]\tvalidation_0-error:0.071338\n",
      "[241]\tvalidation_0-error:0.06879\n",
      "[242]\tvalidation_0-error:0.06879\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[243]\tvalidation_0-error:0.06879\n",
      "[244]\tvalidation_0-error:0.070064\n",
      "[245]\tvalidation_0-error:0.070064\n",
      "[246]\tvalidation_0-error:0.06879\n",
      "[247]\tvalidation_0-error:0.06879\n",
      "[248]\tvalidation_0-error:0.06879\n",
      "[249]\tvalidation_0-error:0.06879\n",
      "[250]\tvalidation_0-error:0.06879\n",
      "[251]\tvalidation_0-error:0.06879\n",
      "[252]\tvalidation_0-error:0.06879\n",
      "[253]\tvalidation_0-error:0.06879\n",
      "[254]\tvalidation_0-error:0.06879\n",
      "[255]\tvalidation_0-error:0.06879\n",
      "[256]\tvalidation_0-error:0.067516\n",
      "[257]\tvalidation_0-error:0.067516\n",
      "[258]\tvalidation_0-error:0.06879\n",
      "[259]\tvalidation_0-error:0.066242\n",
      "[260]\tvalidation_0-error:0.066242\n",
      "[261]\tvalidation_0-error:0.064968\n",
      "[262]\tvalidation_0-error:0.064968\n",
      "[263]\tvalidation_0-error:0.072611\n",
      "[264]\tvalidation_0-error:0.06879\n",
      "[265]\tvalidation_0-error:0.06879\n",
      "[266]\tvalidation_0-error:0.066242\n",
      "[267]\tvalidation_0-error:0.070064\n",
      "[268]\tvalidation_0-error:0.072611\n",
      "[269]\tvalidation_0-error:0.06879\n",
      "[270]\tvalidation_0-error:0.072611\n",
      "[271]\tvalidation_0-error:0.070064\n",
      "[272]\tvalidation_0-error:0.070064\n",
      "[273]\tvalidation_0-error:0.070064\n",
      "[274]\tvalidation_0-error:0.070064\n",
      "[275]\tvalidation_0-error:0.070064\n",
      "[276]\tvalidation_0-error:0.067516\n",
      "[277]\tvalidation_0-error:0.063694\n",
      "[278]\tvalidation_0-error:0.063694\n",
      "[279]\tvalidation_0-error:0.063694\n",
      "[280]\tvalidation_0-error:0.067516\n",
      "[281]\tvalidation_0-error:0.063694\n",
      "[282]\tvalidation_0-error:0.063694\n",
      "[283]\tvalidation_0-error:0.063694\n",
      "[284]\tvalidation_0-error:0.063694\n",
      "[285]\tvalidation_0-error:0.063694\n",
      "[286]\tvalidation_0-error:0.067516\n",
      "[287]\tvalidation_0-error:0.066242\n",
      "[288]\tvalidation_0-error:0.066242\n",
      "[289]\tvalidation_0-error:0.066242\n",
      "[290]\tvalidation_0-error:0.063694\n",
      "[291]\tvalidation_0-error:0.066242\n",
      "[292]\tvalidation_0-error:0.063694\n",
      "[293]\tvalidation_0-error:0.066242\n",
      "[294]\tvalidation_0-error:0.066242\n",
      "[295]\tvalidation_0-error:0.067516\n",
      "[296]\tvalidation_0-error:0.066242\n",
      "[297]\tvalidation_0-error:0.067516\n",
      "[298]\tvalidation_0-error:0.067516\n",
      "[299]\tvalidation_0-error:0.067516\n",
      "[300]\tvalidation_0-error:0.067516\n",
      "[301]\tvalidation_0-error:0.067516\n",
      "[302]\tvalidation_0-error:0.064968\n",
      "[303]\tvalidation_0-error:0.064968\n",
      "[304]\tvalidation_0-error:0.063694\n",
      "[305]\tvalidation_0-error:0.063694\n",
      "[306]\tvalidation_0-error:0.063694\n",
      "[307]\tvalidation_0-error:0.063694\n",
      "[308]\tvalidation_0-error:0.067516\n",
      "[309]\tvalidation_0-error:0.063694\n",
      "[310]\tvalidation_0-error:0.063694\n",
      "[311]\tvalidation_0-error:0.067516\n",
      "[312]\tvalidation_0-error:0.067516\n",
      "[313]\tvalidation_0-error:0.067516\n",
      "[314]\tvalidation_0-error:0.066242\n",
      "[315]\tvalidation_0-error:0.066242\n",
      "[316]\tvalidation_0-error:0.071338\n",
      "[317]\tvalidation_0-error:0.071338\n",
      "[318]\tvalidation_0-error:0.067516\n",
      "[319]\tvalidation_0-error:0.067516\n",
      "[320]\tvalidation_0-error:0.067516\n",
      "[321]\tvalidation_0-error:0.067516\n",
      "[322]\tvalidation_0-error:0.067516\n",
      "[323]\tvalidation_0-error:0.067516\n",
      "[324]\tvalidation_0-error:0.067516\n",
      "[325]\tvalidation_0-error:0.064968\n",
      "[326]\tvalidation_0-error:0.06879\n",
      "[327]\tvalidation_0-error:0.06242\n",
      "[328]\tvalidation_0-error:0.066242\n",
      "[329]\tvalidation_0-error:0.06242\n",
      "[330]\tvalidation_0-error:0.064968\n",
      "[331]\tvalidation_0-error:0.06242\n",
      "[332]\tvalidation_0-error:0.067516\n",
      "[333]\tvalidation_0-error:0.064968\n",
      "[334]\tvalidation_0-error:0.064968\n",
      "[335]\tvalidation_0-error:0.066242\n",
      "[336]\tvalidation_0-error:0.066242\n",
      "[337]\tvalidation_0-error:0.064968\n",
      "[338]\tvalidation_0-error:0.064968\n",
      "[339]\tvalidation_0-error:0.064968\n",
      "[340]\tvalidation_0-error:0.064968\n",
      "[341]\tvalidation_0-error:0.064968\n",
      "[342]\tvalidation_0-error:0.064968\n",
      "[343]\tvalidation_0-error:0.064968\n",
      "[344]\tvalidation_0-error:0.064968\n",
      "[345]\tvalidation_0-error:0.067516\n",
      "[346]\tvalidation_0-error:0.067516\n",
      "[347]\tvalidation_0-error:0.067516\n",
      "[348]\tvalidation_0-error:0.067516\n",
      "[349]\tvalidation_0-error:0.067516\n",
      "[350]\tvalidation_0-error:0.071338\n",
      "[351]\tvalidation_0-error:0.067516\n",
      "[352]\tvalidation_0-error:0.064968\n",
      "[353]\tvalidation_0-error:0.064968\n",
      "[354]\tvalidation_0-error:0.064968\n",
      "[355]\tvalidation_0-error:0.064968\n",
      "[356]\tvalidation_0-error:0.064968\n",
      "[357]\tvalidation_0-error:0.064968\n",
      "[358]\tvalidation_0-error:0.064968\n",
      "[359]\tvalidation_0-error:0.064968\n",
      "[360]\tvalidation_0-error:0.064968\n",
      "[361]\tvalidation_0-error:0.067516\n",
      "[362]\tvalidation_0-error:0.067516\n",
      "[363]\tvalidation_0-error:0.064968\n",
      "[364]\tvalidation_0-error:0.064968\n",
      "[365]\tvalidation_0-error:0.06242\n",
      "[366]\tvalidation_0-error:0.064968\n",
      "[367]\tvalidation_0-error:0.067516\n",
      "[368]\tvalidation_0-error:0.067516\n",
      "[369]\tvalidation_0-error:0.064968\n",
      "[370]\tvalidation_0-error:0.061146\n",
      "[371]\tvalidation_0-error:0.066242\n",
      "[372]\tvalidation_0-error:0.058599\n",
      "[373]\tvalidation_0-error:0.058599\n",
      "[374]\tvalidation_0-error:0.058599\n",
      "[375]\tvalidation_0-error:0.06242\n",
      "[376]\tvalidation_0-error:0.06242\n",
      "[377]\tvalidation_0-error:0.06242\n",
      "[378]\tvalidation_0-error:0.06242\n",
      "[379]\tvalidation_0-error:0.06242\n",
      "[380]\tvalidation_0-error:0.061146\n",
      "[381]\tvalidation_0-error:0.064968\n",
      "[382]\tvalidation_0-error:0.06242\n",
      "[383]\tvalidation_0-error:0.064968\n",
      "[384]\tvalidation_0-error:0.064968\n",
      "[385]\tvalidation_0-error:0.06242\n",
      "[386]\tvalidation_0-error:0.064968\n",
      "[387]\tvalidation_0-error:0.06242\n",
      "[388]\tvalidation_0-error:0.06242\n",
      "[389]\tvalidation_0-error:0.06242\n",
      "[390]\tvalidation_0-error:0.06242\n",
      "[391]\tvalidation_0-error:0.06242\n",
      "[392]\tvalidation_0-error:0.06242\n",
      "[393]\tvalidation_0-error:0.06242\n",
      "[394]\tvalidation_0-error:0.066242\n",
      "[395]\tvalidation_0-error:0.059873\n",
      "[396]\tvalidation_0-error:0.066242\n",
      "[397]\tvalidation_0-error:0.063694\n",
      "[398]\tvalidation_0-error:0.057325\n",
      "[399]\tvalidation_0-error:0.06242\n",
      "[400]\tvalidation_0-error:0.057325\n",
      "[401]\tvalidation_0-error:0.059873\n",
      "[402]\tvalidation_0-error:0.059873\n",
      "[403]\tvalidation_0-error:0.059873\n",
      "[404]\tvalidation_0-error:0.063694\n",
      "[405]\tvalidation_0-error:0.059873\n",
      "[406]\tvalidation_0-error:0.057325\n",
      "[407]\tvalidation_0-error:0.057325\n",
      "[408]\tvalidation_0-error:0.061146\n",
      "[409]\tvalidation_0-error:0.061146\n",
      "[410]\tvalidation_0-error:0.061146\n",
      "[411]\tvalidation_0-error:0.061146\n",
      "[412]\tvalidation_0-error:0.061146\n",
      "[413]\tvalidation_0-error:0.061146\n",
      "[414]\tvalidation_0-error:0.063694\n",
      "[415]\tvalidation_0-error:0.063694\n",
      "[416]\tvalidation_0-error:0.061146\n",
      "[417]\tvalidation_0-error:0.061146\n",
      "[418]\tvalidation_0-error:0.061146\n",
      "[419]\tvalidation_0-error:0.061146\n",
      "[420]\tvalidation_0-error:0.061146\n",
      "[421]\tvalidation_0-error:0.061146\n",
      "[422]\tvalidation_0-error:0.061146\n",
      "[423]\tvalidation_0-error:0.058599\n",
      "[424]\tvalidation_0-error:0.058599\n",
      "[425]\tvalidation_0-error:0.061146\n",
      "[426]\tvalidation_0-error:0.061146\n",
      "[427]\tvalidation_0-error:0.061146\n",
      "[428]\tvalidation_0-error:0.061146\n",
      "[429]\tvalidation_0-error:0.061146\n",
      "[430]\tvalidation_0-error:0.057325\n",
      "[431]\tvalidation_0-error:0.054777\n",
      "[432]\tvalidation_0-error:0.058599\n",
      "[433]\tvalidation_0-error:0.058599\n",
      "[434]\tvalidation_0-error:0.061146\n",
      "[435]\tvalidation_0-error:0.058599\n",
      "[436]\tvalidation_0-error:0.061146\n",
      "[437]\tvalidation_0-error:0.063694\n",
      "[438]\tvalidation_0-error:0.061146\n",
      "[439]\tvalidation_0-error:0.061146\n",
      "[440]\tvalidation_0-error:0.061146\n",
      "[441]\tvalidation_0-error:0.061146\n",
      "[442]\tvalidation_0-error:0.059873\n",
      "[443]\tvalidation_0-error:0.061146\n",
      "[444]\tvalidation_0-error:0.061146\n",
      "[445]\tvalidation_0-error:0.057325\n",
      "[446]\tvalidation_0-error:0.058599\n",
      "[447]\tvalidation_0-error:0.057325\n",
      "[448]\tvalidation_0-error:0.061146\n",
      "[449]\tvalidation_0-error:0.061146\n",
      "[450]\tvalidation_0-error:0.057325\n",
      "[451]\tvalidation_0-error:0.057325\n",
      "[452]\tvalidation_0-error:0.061146\n",
      "[453]\tvalidation_0-error:0.057325\n",
      "[454]\tvalidation_0-error:0.061146\n",
      "[455]\tvalidation_0-error:0.061146\n",
      "[456]\tvalidation_0-error:0.063694\n",
      "[457]\tvalidation_0-error:0.058599\n",
      "[458]\tvalidation_0-error:0.061146\n",
      "[459]\tvalidation_0-error:0.058599\n",
      "[460]\tvalidation_0-error:0.061146\n",
      "[461]\tvalidation_0-error:0.064968\n",
      "[462]\tvalidation_0-error:0.064968\n",
      "[463]\tvalidation_0-error:0.06242\n",
      "[464]\tvalidation_0-error:0.054777\n",
      "[465]\tvalidation_0-error:0.054777\n",
      "[466]\tvalidation_0-error:0.054777\n",
      "[467]\tvalidation_0-error:0.054777\n",
      "[468]\tvalidation_0-error:0.057325\n",
      "[469]\tvalidation_0-error:0.057325\n",
      "[470]\tvalidation_0-error:0.057325\n",
      "[471]\tvalidation_0-error:0.057325\n",
      "[472]\tvalidation_0-error:0.056051\n",
      "[473]\tvalidation_0-error:0.056051\n",
      "[474]\tvalidation_0-error:0.056051\n",
      "[475]\tvalidation_0-error:0.054777\n",
      "[476]\tvalidation_0-error:0.059873\n",
      "[477]\tvalidation_0-error:0.056051\n",
      "[478]\tvalidation_0-error:0.056051\n",
      "[479]\tvalidation_0-error:0.054777\n",
      "[480]\tvalidation_0-error:0.054777\n",
      "[481]\tvalidation_0-error:0.054777\n",
      "[482]\tvalidation_0-error:0.054777\n",
      "[483]\tvalidation_0-error:0.054777\n",
      "[484]\tvalidation_0-error:0.054777\n",
      "[485]\tvalidation_0-error:0.054777\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[486]\tvalidation_0-error:0.054777\n",
      "[487]\tvalidation_0-error:0.054777\n",
      "[488]\tvalidation_0-error:0.054777\n",
      "[489]\tvalidation_0-error:0.054777\n",
      "[490]\tvalidation_0-error:0.054777\n",
      "[491]\tvalidation_0-error:0.054777\n",
      "[492]\tvalidation_0-error:0.054777\n",
      "[493]\tvalidation_0-error:0.054777\n",
      "[494]\tvalidation_0-error:0.054777\n",
      "[495]\tvalidation_0-error:0.054777\n",
      "[496]\tvalidation_0-error:0.054777\n",
      "[497]\tvalidation_0-error:0.054777\n",
      "[498]\tvalidation_0-error:0.053503\n",
      "[499]\tvalidation_0-error:0.053503\n",
      "[500]\tvalidation_0-error:0.053503\n",
      "[501]\tvalidation_0-error:0.054777\n",
      "[502]\tvalidation_0-error:0.054777\n",
      "[503]\tvalidation_0-error:0.056051\n",
      "[504]\tvalidation_0-error:0.059873\n",
      "[505]\tvalidation_0-error:0.056051\n",
      "[506]\tvalidation_0-error:0.058599\n",
      "[507]\tvalidation_0-error:0.058599\n",
      "[508]\tvalidation_0-error:0.058599\n",
      "[509]\tvalidation_0-error:0.054777\n",
      "[510]\tvalidation_0-error:0.054777\n",
      "[511]\tvalidation_0-error:0.054777\n",
      "[512]\tvalidation_0-error:0.054777\n",
      "[513]\tvalidation_0-error:0.054777\n",
      "[514]\tvalidation_0-error:0.057325\n",
      "[515]\tvalidation_0-error:0.057325\n",
      "[516]\tvalidation_0-error:0.057325\n",
      "[517]\tvalidation_0-error:0.058599\n",
      "[518]\tvalidation_0-error:0.057325\n",
      "[519]\tvalidation_0-error:0.058599\n",
      "[520]\tvalidation_0-error:0.058599\n",
      "[521]\tvalidation_0-error:0.058599\n",
      "[522]\tvalidation_0-error:0.058599\n",
      "[523]\tvalidation_0-error:0.058599\n",
      "[524]\tvalidation_0-error:0.058599\n",
      "[525]\tvalidation_0-error:0.058599\n",
      "[526]\tvalidation_0-error:0.058599\n",
      "[527]\tvalidation_0-error:0.058599\n",
      "[528]\tvalidation_0-error:0.058599\n",
      "[529]\tvalidation_0-error:0.058599\n",
      "[530]\tvalidation_0-error:0.058599\n",
      "[531]\tvalidation_0-error:0.058599\n",
      "[532]\tvalidation_0-error:0.058599\n",
      "[533]\tvalidation_0-error:0.058599\n",
      "[534]\tvalidation_0-error:0.058599\n",
      "[535]\tvalidation_0-error:0.058599\n",
      "[536]\tvalidation_0-error:0.058599\n",
      "[537]\tvalidation_0-error:0.058599\n",
      "[538]\tvalidation_0-error:0.058599\n",
      "[539]\tvalidation_0-error:0.058599\n",
      "[540]\tvalidation_0-error:0.058599\n",
      "[541]\tvalidation_0-error:0.058599\n",
      "[542]\tvalidation_0-error:0.058599\n",
      "[543]\tvalidation_0-error:0.058599\n",
      "[544]\tvalidation_0-error:0.058599\n",
      "[545]\tvalidation_0-error:0.058599\n",
      "[546]\tvalidation_0-error:0.058599\n",
      "[547]\tvalidation_0-error:0.058599\n",
      "[548]\tvalidation_0-error:0.058599\n",
      "[549]\tvalidation_0-error:0.058599\n",
      "[550]\tvalidation_0-error:0.058599\n",
      "[551]\tvalidation_0-error:0.058599\n",
      "[552]\tvalidation_0-error:0.058599\n",
      "[553]\tvalidation_0-error:0.058599\n",
      "[554]\tvalidation_0-error:0.058599\n",
      "[555]\tvalidation_0-error:0.058599\n",
      "[556]\tvalidation_0-error:0.058599\n",
      "[557]\tvalidation_0-error:0.058599\n",
      "[558]\tvalidation_0-error:0.058599\n",
      "[559]\tvalidation_0-error:0.054777\n",
      "[560]\tvalidation_0-error:0.058599\n",
      "[561]\tvalidation_0-error:0.058599\n",
      "[562]\tvalidation_0-error:0.058599\n",
      "[563]\tvalidation_0-error:0.058599\n",
      "[564]\tvalidation_0-error:0.058599\n",
      "[565]\tvalidation_0-error:0.058599\n",
      "[566]\tvalidation_0-error:0.058599\n",
      "[567]\tvalidation_0-error:0.058599\n",
      "[568]\tvalidation_0-error:0.058599\n",
      "[569]\tvalidation_0-error:0.058599\n",
      "[570]\tvalidation_0-error:0.059873\n",
      "[571]\tvalidation_0-error:0.059873\n",
      "[572]\tvalidation_0-error:0.058599\n",
      "[573]\tvalidation_0-error:0.058599\n",
      "[574]\tvalidation_0-error:0.058599\n",
      "[575]\tvalidation_0-error:0.058599\n",
      "[576]\tvalidation_0-error:0.058599\n",
      "[577]\tvalidation_0-error:0.059873\n",
      "[578]\tvalidation_0-error:0.059873\n",
      "[579]\tvalidation_0-error:0.059873\n",
      "[580]\tvalidation_0-error:0.058599\n",
      "[581]\tvalidation_0-error:0.058599\n",
      "[582]\tvalidation_0-error:0.058599\n",
      "[583]\tvalidation_0-error:0.058599\n",
      "[584]\tvalidation_0-error:0.058599\n",
      "[585]\tvalidation_0-error:0.058599\n",
      "[586]\tvalidation_0-error:0.058599\n",
      "[587]\tvalidation_0-error:0.058599\n",
      "[588]\tvalidation_0-error:0.058599\n",
      "[589]\tvalidation_0-error:0.058599\n",
      "[590]\tvalidation_0-error:0.058599\n",
      "[591]\tvalidation_0-error:0.054777\n",
      "[592]\tvalidation_0-error:0.058599\n",
      "[593]\tvalidation_0-error:0.054777\n",
      "[594]\tvalidation_0-error:0.054777\n",
      "[595]\tvalidation_0-error:0.054777\n",
      "[596]\tvalidation_0-error:0.054777\n",
      "[597]\tvalidation_0-error:0.054777\n",
      "[598]\tvalidation_0-error:0.054777\n",
      "[599]\tvalidation_0-error:0.054777\n",
      "[600]\tvalidation_0-error:0.056051\n",
      "[601]\tvalidation_0-error:0.056051\n",
      "[602]\tvalidation_0-error:0.054777\n",
      "[603]\tvalidation_0-error:0.054777\n",
      "[604]\tvalidation_0-error:0.054777\n",
      "[605]\tvalidation_0-error:0.053503\n",
      "[606]\tvalidation_0-error:0.058599\n",
      "[607]\tvalidation_0-error:0.059873\n",
      "[608]\tvalidation_0-error:0.059873\n",
      "[609]\tvalidation_0-error:0.054777\n",
      "[610]\tvalidation_0-error:0.053503\n",
      "[611]\tvalidation_0-error:0.057325\n",
      "[612]\tvalidation_0-error:0.054777\n",
      "[613]\tvalidation_0-error:0.056051\n",
      "[614]\tvalidation_0-error:0.056051\n",
      "[615]\tvalidation_0-error:0.056051\n",
      "[616]\tvalidation_0-error:0.058599\n",
      "[617]\tvalidation_0-error:0.056051\n",
      "[618]\tvalidation_0-error:0.056051\n",
      "[619]\tvalidation_0-error:0.057325\n",
      "[620]\tvalidation_0-error:0.058599\n",
      "[621]\tvalidation_0-error:0.058599\n",
      "[622]\tvalidation_0-error:0.054777\n",
      "[623]\tvalidation_0-error:0.054777\n",
      "[624]\tvalidation_0-error:0.054777\n",
      "[625]\tvalidation_0-error:0.054777\n",
      "[626]\tvalidation_0-error:0.054777\n",
      "[627]\tvalidation_0-error:0.054777\n",
      "[628]\tvalidation_0-error:0.054777\n",
      "[629]\tvalidation_0-error:0.053503\n",
      "[630]\tvalidation_0-error:0.053503\n",
      "[631]\tvalidation_0-error:0.053503\n",
      "[632]\tvalidation_0-error:0.053503\n",
      "[633]\tvalidation_0-error:0.054777\n",
      "[634]\tvalidation_0-error:0.054777\n",
      "[635]\tvalidation_0-error:0.054777\n",
      "[636]\tvalidation_0-error:0.058599\n",
      "[637]\tvalidation_0-error:0.058599\n",
      "[638]\tvalidation_0-error:0.057325\n",
      "[639]\tvalidation_0-error:0.057325\n",
      "[640]\tvalidation_0-error:0.057325\n",
      "[641]\tvalidation_0-error:0.054777\n",
      "[642]\tvalidation_0-error:0.054777\n",
      "[643]\tvalidation_0-error:0.054777\n",
      "[644]\tvalidation_0-error:0.049682\n",
      "[645]\tvalidation_0-error:0.053503\n",
      "[646]\tvalidation_0-error:0.049682\n",
      "[647]\tvalidation_0-error:0.053503\n",
      "[648]\tvalidation_0-error:0.049682\n",
      "[649]\tvalidation_0-error:0.049682\n",
      "[650]\tvalidation_0-error:0.053503\n",
      "[651]\tvalidation_0-error:0.053503\n",
      "[652]\tvalidation_0-error:0.053503\n",
      "[653]\tvalidation_0-error:0.053503\n",
      "[654]\tvalidation_0-error:0.053503\n",
      "[655]\tvalidation_0-error:0.053503\n",
      "[656]\tvalidation_0-error:0.053503\n",
      "[657]\tvalidation_0-error:0.053503\n",
      "[658]\tvalidation_0-error:0.053503\n",
      "[659]\tvalidation_0-error:0.053503\n",
      "[660]\tvalidation_0-error:0.053503\n",
      "[661]\tvalidation_0-error:0.053503\n",
      "[662]\tvalidation_0-error:0.053503\n",
      "[663]\tvalidation_0-error:0.053503\n",
      "[664]\tvalidation_0-error:0.053503\n",
      "[665]\tvalidation_0-error:0.053503\n",
      "[666]\tvalidation_0-error:0.053503\n",
      "[667]\tvalidation_0-error:0.053503\n",
      "[668]\tvalidation_0-error:0.053503\n",
      "[669]\tvalidation_0-error:0.053503\n",
      "[670]\tvalidation_0-error:0.057325\n",
      "[671]\tvalidation_0-error:0.053503\n",
      "[672]\tvalidation_0-error:0.057325\n",
      "[673]\tvalidation_0-error:0.057325\n",
      "[674]\tvalidation_0-error:0.057325\n",
      "[675]\tvalidation_0-error:0.053503\n",
      "[676]\tvalidation_0-error:0.057325\n",
      "[677]\tvalidation_0-error:0.057325\n",
      "[678]\tvalidation_0-error:0.057325\n",
      "[679]\tvalidation_0-error:0.057325\n",
      "[680]\tvalidation_0-error:0.057325\n",
      "[681]\tvalidation_0-error:0.053503\n",
      "[682]\tvalidation_0-error:0.057325\n",
      "[683]\tvalidation_0-error:0.057325\n",
      "[684]\tvalidation_0-error:0.058599\n",
      "[685]\tvalidation_0-error:0.058599\n",
      "[686]\tvalidation_0-error:0.058599\n",
      "[687]\tvalidation_0-error:0.058599\n",
      "[688]\tvalidation_0-error:0.057325\n",
      "[689]\tvalidation_0-error:0.056051\n",
      "[690]\tvalidation_0-error:0.056051\n",
      "[691]\tvalidation_0-error:0.056051\n",
      "[692]\tvalidation_0-error:0.056051\n",
      "[693]\tvalidation_0-error:0.056051\n",
      "[694]\tvalidation_0-error:0.056051\n",
      "[695]\tvalidation_0-error:0.056051\n",
      "[696]\tvalidation_0-error:0.056051\n",
      "[697]\tvalidation_0-error:0.056051\n",
      "[698]\tvalidation_0-error:0.057325\n",
      "[699]\tvalidation_0-error:0.057325\n",
      "[700]\tvalidation_0-error:0.058599\n",
      "[701]\tvalidation_0-error:0.058599\n",
      "[702]\tvalidation_0-error:0.058599\n",
      "[703]\tvalidation_0-error:0.058599\n",
      "[704]\tvalidation_0-error:0.057325\n",
      "[705]\tvalidation_0-error:0.057325\n",
      "[706]\tvalidation_0-error:0.057325\n",
      "[707]\tvalidation_0-error:0.057325\n",
      "[708]\tvalidation_0-error:0.057325\n",
      "[709]\tvalidation_0-error:0.058599\n",
      "[710]\tvalidation_0-error:0.058599\n",
      "[711]\tvalidation_0-error:0.058599\n",
      "[712]\tvalidation_0-error:0.058599\n",
      "[713]\tvalidation_0-error:0.058599\n",
      "[714]\tvalidation_0-error:0.058599\n",
      "[715]\tvalidation_0-error:0.057325\n",
      "[716]\tvalidation_0-error:0.057325\n",
      "[717]\tvalidation_0-error:0.052229\n",
      "[718]\tvalidation_0-error:0.052229\n",
      "[719]\tvalidation_0-error:0.048408\n",
      "[720]\tvalidation_0-error:0.053503\n",
      "[721]\tvalidation_0-error:0.053503\n",
      "[722]\tvalidation_0-error:0.053503\n",
      "[723]\tvalidation_0-error:0.057325\n",
      "[724]\tvalidation_0-error:0.057325\n",
      "[725]\tvalidation_0-error:0.053503\n",
      "[726]\tvalidation_0-error:0.057325\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[727]\tvalidation_0-error:0.057325\n",
      "[728]\tvalidation_0-error:0.053503\n",
      "[729]\tvalidation_0-error:0.057325\n",
      "[730]\tvalidation_0-error:0.057325\n",
      "[731]\tvalidation_0-error:0.057325\n",
      "[732]\tvalidation_0-error:0.057325\n",
      "[733]\tvalidation_0-error:0.057325\n",
      "[734]\tvalidation_0-error:0.053503\n",
      "[735]\tvalidation_0-error:0.053503\n",
      "[736]\tvalidation_0-error:0.057325\n",
      "[737]\tvalidation_0-error:0.053503\n",
      "[738]\tvalidation_0-error:0.053503\n",
      "[739]\tvalidation_0-error:0.057325\n",
      "[740]\tvalidation_0-error:0.053503\n",
      "[741]\tvalidation_0-error:0.056051\n",
      "[742]\tvalidation_0-error:0.056051\n",
      "[743]\tvalidation_0-error:0.056051\n",
      "[744]\tvalidation_0-error:0.056051\n",
      "[745]\tvalidation_0-error:0.056051\n",
      "[746]\tvalidation_0-error:0.056051\n",
      "[747]\tvalidation_0-error:0.056051\n",
      "[748]\tvalidation_0-error:0.056051\n",
      "[749]\tvalidation_0-error:0.056051\n",
      "[750]\tvalidation_0-error:0.056051\n",
      "[751]\tvalidation_0-error:0.056051\n",
      "[752]\tvalidation_0-error:0.056051\n",
      "[753]\tvalidation_0-error:0.056051\n",
      "[754]\tvalidation_0-error:0.056051\n",
      "[755]\tvalidation_0-error:0.056051\n",
      "[756]\tvalidation_0-error:0.056051\n",
      "[757]\tvalidation_0-error:0.056051\n",
      "[758]\tvalidation_0-error:0.052229\n",
      "[759]\tvalidation_0-error:0.056051\n",
      "[760]\tvalidation_0-error:0.056051\n",
      "[761]\tvalidation_0-error:0.056051\n",
      "[762]\tvalidation_0-error:0.056051\n",
      "[763]\tvalidation_0-error:0.056051\n",
      "[764]\tvalidation_0-error:0.056051\n",
      "[765]\tvalidation_0-error:0.056051\n",
      "[766]\tvalidation_0-error:0.056051\n",
      "[767]\tvalidation_0-error:0.056051\n",
      "[768]\tvalidation_0-error:0.056051\n",
      "[769]\tvalidation_0-error:0.056051\n",
      "[770]\tvalidation_0-error:0.053503\n",
      "[771]\tvalidation_0-error:0.057325\n",
      "[772]\tvalidation_0-error:0.053503\n",
      "[773]\tvalidation_0-error:0.057325\n",
      "[774]\tvalidation_0-error:0.057325\n",
      "[775]\tvalidation_0-error:0.057325\n",
      "[776]\tvalidation_0-error:0.057325\n",
      "[777]\tvalidation_0-error:0.057325\n",
      "[778]\tvalidation_0-error:0.057325\n",
      "[779]\tvalidation_0-error:0.057325\n",
      "[780]\tvalidation_0-error:0.057325\n",
      "[781]\tvalidation_0-error:0.056051\n",
      "[782]\tvalidation_0-error:0.056051\n",
      "[783]\tvalidation_0-error:0.057325\n",
      "[784]\tvalidation_0-error:0.057325\n",
      "[785]\tvalidation_0-error:0.057325\n",
      "[786]\tvalidation_0-error:0.057325\n",
      "[787]\tvalidation_0-error:0.057325\n",
      "[788]\tvalidation_0-error:0.057325\n",
      "[789]\tvalidation_0-error:0.057325\n",
      "[790]\tvalidation_0-error:0.057325\n",
      "[791]\tvalidation_0-error:0.056051\n",
      "[792]\tvalidation_0-error:0.056051\n",
      "[793]\tvalidation_0-error:0.056051\n",
      "[794]\tvalidation_0-error:0.052229\n",
      "[795]\tvalidation_0-error:0.052229\n",
      "[796]\tvalidation_0-error:0.052229\n",
      "[797]\tvalidation_0-error:0.052229\n",
      "[798]\tvalidation_0-error:0.052229\n",
      "[799]\tvalidation_0-error:0.052229\n",
      "[800]\tvalidation_0-error:0.052229\n",
      "[801]\tvalidation_0-error:0.052229\n",
      "[802]\tvalidation_0-error:0.052229\n",
      "[803]\tvalidation_0-error:0.052229\n",
      "[804]\tvalidation_0-error:0.052229\n",
      "[805]\tvalidation_0-error:0.052229\n",
      "[806]\tvalidation_0-error:0.052229\n",
      "[807]\tvalidation_0-error:0.052229\n",
      "[808]\tvalidation_0-error:0.052229\n",
      "[809]\tvalidation_0-error:0.053503\n",
      "[810]\tvalidation_0-error:0.053503\n",
      "[811]\tvalidation_0-error:0.053503\n",
      "[812]\tvalidation_0-error:0.052229\n",
      "[813]\tvalidation_0-error:0.052229\n",
      "[814]\tvalidation_0-error:0.052229\n",
      "[815]\tvalidation_0-error:0.052229\n",
      "[816]\tvalidation_0-error:0.053503\n",
      "[817]\tvalidation_0-error:0.053503\n",
      "[818]\tvalidation_0-error:0.053503\n",
      "[819]\tvalidation_0-error:0.053503\n",
      "[820]\tvalidation_0-error:0.053503\n",
      "[821]\tvalidation_0-error:0.052229\n",
      "[822]\tvalidation_0-error:0.052229\n",
      "[823]\tvalidation_0-error:0.052229\n",
      "[824]\tvalidation_0-error:0.052229\n",
      "[825]\tvalidation_0-error:0.052229\n",
      "[826]\tvalidation_0-error:0.052229\n",
      "[827]\tvalidation_0-error:0.052229\n",
      "[828]\tvalidation_0-error:0.052229\n",
      "[829]\tvalidation_0-error:0.052229\n",
      "[830]\tvalidation_0-error:0.053503\n",
      "[831]\tvalidation_0-error:0.053503\n",
      "[832]\tvalidation_0-error:0.053503\n",
      "[833]\tvalidation_0-error:0.054777\n",
      "[834]\tvalidation_0-error:0.054777\n",
      "[835]\tvalidation_0-error:0.054777\n",
      "[836]\tvalidation_0-error:0.052229\n",
      "[837]\tvalidation_0-error:0.052229\n",
      "[838]\tvalidation_0-error:0.052229\n",
      "[839]\tvalidation_0-error:0.052229\n",
      "[840]\tvalidation_0-error:0.053503\n",
      "[841]\tvalidation_0-error:0.052229\n",
      "[842]\tvalidation_0-error:0.052229\n",
      "[843]\tvalidation_0-error:0.052229\n",
      "[844]\tvalidation_0-error:0.052229\n",
      "[845]\tvalidation_0-error:0.052229\n",
      "[846]\tvalidation_0-error:0.053503\n",
      "[847]\tvalidation_0-error:0.053503\n",
      "[848]\tvalidation_0-error:0.052229\n",
      "[849]\tvalidation_0-error:0.052229\n",
      "[850]\tvalidation_0-error:0.048408\n",
      "[851]\tvalidation_0-error:0.052229\n",
      "[852]\tvalidation_0-error:0.052229\n",
      "[853]\tvalidation_0-error:0.052229\n",
      "[854]\tvalidation_0-error:0.048408\n",
      "[855]\tvalidation_0-error:0.052229\n",
      "[856]\tvalidation_0-error:0.052229\n",
      "[857]\tvalidation_0-error:0.052229\n",
      "[858]\tvalidation_0-error:0.052229\n",
      "[859]\tvalidation_0-error:0.052229\n",
      "[860]\tvalidation_0-error:0.052229\n",
      "[861]\tvalidation_0-error:0.056051\n",
      "[862]\tvalidation_0-error:0.056051\n",
      "[863]\tvalidation_0-error:0.056051\n",
      "[864]\tvalidation_0-error:0.056051\n",
      "[865]\tvalidation_0-error:0.056051\n",
      "[866]\tvalidation_0-error:0.057325\n",
      "[867]\tvalidation_0-error:0.057325\n",
      "[868]\tvalidation_0-error:0.057325\n",
      "[869]\tvalidation_0-error:0.057325\n",
      "[870]\tvalidation_0-error:0.057325\n",
      "[871]\tvalidation_0-error:0.057325\n",
      "[872]\tvalidation_0-error:0.053503\n",
      "[873]\tvalidation_0-error:0.057325\n",
      "[874]\tvalidation_0-error:0.057325\n",
      "[875]\tvalidation_0-error:0.053503\n",
      "[876]\tvalidation_0-error:0.057325\n",
      "[877]\tvalidation_0-error:0.052229\n",
      "[878]\tvalidation_0-error:0.056051\n",
      "[879]\tvalidation_0-error:0.052229\n",
      "[880]\tvalidation_0-error:0.052229\n",
      "[881]\tvalidation_0-error:0.052229\n",
      "[882]\tvalidation_0-error:0.056051\n",
      "[883]\tvalidation_0-error:0.056051\n",
      "[884]\tvalidation_0-error:0.056051\n",
      "[885]\tvalidation_0-error:0.056051\n",
      "[886]\tvalidation_0-error:0.057325\n",
      "[887]\tvalidation_0-error:0.057325\n",
      "[888]\tvalidation_0-error:0.057325\n",
      "[889]\tvalidation_0-error:0.057325\n",
      "[890]\tvalidation_0-error:0.056051\n",
      "[891]\tvalidation_0-error:0.056051\n",
      "[892]\tvalidation_0-error:0.056051\n",
      "[893]\tvalidation_0-error:0.056051\n",
      "[894]\tvalidation_0-error:0.056051\n",
      "[895]\tvalidation_0-error:0.056051\n",
      "[896]\tvalidation_0-error:0.056051\n",
      "[897]\tvalidation_0-error:0.056051\n",
      "[898]\tvalidation_0-error:0.056051\n",
      "[899]\tvalidation_0-error:0.056051\n",
      "[900]\tvalidation_0-error:0.056051\n",
      "[901]\tvalidation_0-error:0.056051\n",
      "[902]\tvalidation_0-error:0.052229\n",
      "[903]\tvalidation_0-error:0.056051\n",
      "[904]\tvalidation_0-error:0.056051\n",
      "[905]\tvalidation_0-error:0.052229\n",
      "[906]\tvalidation_0-error:0.052229\n",
      "[907]\tvalidation_0-error:0.052229\n",
      "[908]\tvalidation_0-error:0.052229\n",
      "[909]\tvalidation_0-error:0.052229\n",
      "[910]\tvalidation_0-error:0.052229\n",
      "[911]\tvalidation_0-error:0.053503\n",
      "[912]\tvalidation_0-error:0.056051\n",
      "[913]\tvalidation_0-error:0.057325\n",
      "[914]\tvalidation_0-error:0.057325\n",
      "[915]\tvalidation_0-error:0.057325\n",
      "[916]\tvalidation_0-error:0.057325\n",
      "[917]\tvalidation_0-error:0.052229\n",
      "[918]\tvalidation_0-error:0.053503\n",
      "[919]\tvalidation_0-error:0.053503\n",
      "Stopping. Best iteration:\n",
      "[719]\tvalidation_0-error:0.048408\n",
      "\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "XGBClassifier(base_score=0.5, booster='gbtree', colsample_bylevel=0.7,\n",
       "       colsample_bytree=0.6, gamma=0.1, learning_rate=0.2,\n",
       "       max_delta_step=0, max_depth=15, min_child_weight=0.5, missing=None,\n",
       "       n_estimators=1200, n_jobs=1, nthread=None,\n",
       "       objective='binary:logistic', random_state=42, reg_alpha=0.003,\n",
       "       reg_lambda=0.04, scale_pos_weight=1, seed=None, silent=1,\n",
       "       subsample=0.7)"
      ]
     },
     "execution_count": 36,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from xgboost import XGBClassifier   \n",
    "# sklearn API 类似于导入的从skearn中导入某个算法，然后再进行实例化即可，初始化算法的时候可以修改默认参数\n",
    "from xgboost import plot_importance\n",
    "x_train_vect=vect.fit_transform(X_train[\"cut_comment\"])\n",
    "x_test_vect= vect.transform(X_test[\"cut_comment\"])\n",
    "clf = XGBClassifier(\n",
    "silent=1 ,#设置成1则没有运行信息输出，最好是设置为0.是否在运行升级时打印消息。\n",
    "# #nthread=4,# cpu 线程数 默认最大\n",
    "learning_rate= 0.20, # 学习率\n",
    "min_child_weight=0.5, \n",
    "# # 这个参数默认是 1，是每个叶子里面 h 的和至少是多少，对正负样本不均衡时的 0-1 分类而言\n",
    "# #，假设 h 在 0.01 附近，min_child_weight 为 1 意味着叶子节点中最少需要包含 100 个样本。\n",
    "# #这个参数非常影响结果，控制叶子节点中二阶导的和的最小值，该参数值越小，越容易 overfitting。\n",
    "gamma=0.1,  # 树的叶子节点上作进一步分区所需的最小损失减少,越大越保守，一般0.1、0.2这样子。\n",
    "subsample=0.7, # 随机采样训练样本 训练实例的子采样比\n",
    "max_depth=15,\n",
    "max_delta_step=0,#最大增量步长，我们允许每个树的权重估计。\n",
    "colsample_bylevel=0.7, # Subsample ratio of columns for each split, in each level.\n",
    "colsample_bytree=0.6, # 生成树时进行的列采样 \n",
    "reg_lambda=0.04,  # 控制模型复杂度的权重值的L2正则化项参数，参数越大，模型越保守\n",
    "reg_alpha=0.003, # L1 正则项参数，参数越大，模型越保守\n",
    "### 正则化是在梯度提升树种没有的，这是xgboost与GB方法的区别之一。\n",
    "scale_pos_weight=1, #如果取值大于0的话，在类别样本不平衡的情况下有助于快速收敛。平衡正负权重=sum(负类样本)/sum(正类样本)\n",
    "# objective= 'reg:logistic', #多分类的问题 指定学习任务和相应的学习目标\n",
    "objective='binary:logistic' ,\n",
    "# #num_class=10, # 类别数，多分类与 multisoftmax 并用\n",
    "n_estimators=1200, #树的个数\n",
    "random_state=42\n",
    "# #eval_metric= 'auc'\n",
    ")\n",
    "evals  = [(x_test_vect, y_test)]\n",
    "clf.fit(x_train_vect,y_train,eval_set=evals,early_stopping_rounds=200)\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 37,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9515923566878981"
      ]
     },
     "execution_count": 37,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "y_pred=clf.predict(x_test_vect)\n",
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 38,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[260,  29],\n",
       "       [  9, 487]], dtype=int64)"
      ]
     },
     "execution_count": 38,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### Xgboost learning  API"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 39,
   "metadata": {},
   "outputs": [],
   "source": [
    "x_train_vect=vect.fit_transform(X_train[\"cut_comment\"])\n",
    "x_test_vect= vect.transform(X_test[\"cut_comment\"])\n",
    "import xgboost as xgb\n",
    "dtrain = xgb.DMatrix(x_train_vect, label=y_train)\n",
    "dtest=xgb.DMatrix(x_test_vect,label=y_test)\n",
    "evals  = [(dtest, 'eval')]\n",
    "param = {'max_depth':15, 'eta':0.2, 'silent':1, 'objective':'binary:logistic', 'gamma':0.1,'colsample_bytree':0.6,\n",
    "        'alpha':0.003,'lambda':0.04, 'colsample_bylevel':0.7,'subsample':0.7,'min_child_weight':0.5,'evals':evals,'early_stopping_rounds':200 }\n",
    "\n",
    "bst = xgb.train(param, dtrain,1200)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 40,
   "metadata": {},
   "outputs": [],
   "source": [
    "# print(bst.best_ntree_limit)\n",
    "y_pred = bst.predict(dtest,ntree_limit=bst.best_ntree_limit)\n",
    "y_pred=[1 if i>0.5 else 0 for i in y_pred]\n",
    "# predictions = [round(value) for value in y_pred]  "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 41,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.95541401273885351"
      ]
     },
     "execution_count": 41,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.accuracy_score(y_test,y_pred)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 42,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([[257,  32],\n",
       "       [  3, 493]], dtype=int64)"
      ]
     },
     "execution_count": 42,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "metrics.confusion_matrix(y_test,y_pred)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.2"
  },
  "toc": {
   "base_numbering": 1,
   "nav_menu": {},
   "number_sections": true,
   "sideBar": true,
   "skip_h1_title": false,
   "title_cell": "Table of Contents",
   "title_sidebar": "Contents",
   "toc_cell": false,
   "toc_position": {
    "height": "calc(100% - 180px)",
    "left": "10px",
    "top": "150px",
    "width": "194px"
   },
   "toc_section_display": true,
   "toc_window_display": true
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
